﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using DataProcessor.Core;
using LumenWorks.Framework.IO.Csv;
using LinqLib.Operators;
using LinqLib.Sequence;

namespace DataProcessor.FashionData
{
    public class FashionDataProcessor
    {
        public IEnumerable<Assignment> Assignments { get; private set; }
        
        public FashionDataProcessor()
        {
            Assignments = new List<Assignment>();
        }

        public FashionDataProcessor LoadAssignments(string amtResultFile)
        {
            Assignments.Concat(new CsvReader(new StreamReader(amtResultFile), true).ToCsvDictionary()
                .Select(a => new Assignment(a)).ToList());

            return this;
        }

        // This method is to create a subset of imageUrls in order to make hit input for a specific subset of images
        public void CreateSubsetImageUrls(string imageUrlFile, string filterFile, string outputFile)
        { 
            var allUrls = File.ReadAllLines(imageUrlFile).ToCsvDictionary()
                .Select(i => new { Category = i["Category"], Url = i["Url"] });

            var filteredUrls = File.ReadAllLines(filterFile).Shuffle().ToList();

            var output = allUrls.Join(filteredUrls, a => a.Url, f => f, (a, f) => a)
                .Select(i => string.Format("{0},{1}", i.Category, i.Url));

            var header = new string[] { "Category,Url" };
            File.WriteAllLines(outputFile, header.Concat(output));                
        }

        public void ShuffleAndSelect(string inputFile, string outputFile)
        {
            File.WriteAllLines(outputFile, File.ReadAllLines(inputFile).Shuffle().Take(200));
        }
        
        public void MergeBatchFiles(string outputFile, string[] inputFiles)
        {
            StreamWriter sw = new StreamWriter(outputFile);
            
            inputFiles.ForEach((i, ix) => 
                File.ReadAllLines(i).Skip(ix == 0 ? 0 : 1).ForEach(l => sw.WriteLine(l)));

            sw.Close();
        }

        public void CheckUploaders(string csvFile, string infoFile, string output)
        {
            var ImageIDs = File.ReadAllLines(csvFile)
                .Select(i => i.Substring(i.LastIndexOf('/') + 1, i.LastIndexOf('_') - i.LastIndexOf('/') - 1));

            var uploaders = new CsvReader(new StreamReader(infoFile), true).ToCsvDictionary()
                .Select(i => new { User = i["OWNER"], ImageId = i["ID"] })
                .Join(ImageIDs, u => u.ImageId, i => i, (u, i) => u)
                .GroupBy(g => g.User)
                .Select(g => new { User = g.Key, Count = g.Count() })
                .OrderByDescending(u => u.Count)
                .Select(i => i.User + "," + i.Count);

            File.WriteAllLines(output, uploaders.ToList());
        }

        public void DetectRejectAssignments(string resultFile)
        {
            Func<Assignment, bool> rejectRule1 = a => a.Related1 == "no" && (a.Prof1 == "yes");
            Func<Assignment, bool> rejectRule2 = a => a.Related2 == "no" && (a.Prof2 == "yes");
            Func<Assignment, bool> rejectRule3 = a => a.Related3 == "no" && (a.Prof3 == "yes");
            Func<Assignment, bool> rejectRule4 = a => a.Related4 == "no" && (a.Prof4 == "yes");

            Func<Assignment, bool> rejectRule5 = a => a.Related1 == "no" && (a.Type1 == "yes");
            Func<Assignment, bool> rejectRule6 = a => a.Related2 == "no" && (a.Type2 == "yes");
            Func<Assignment, bool> rejectRule7 = a => a.Related3 == "no" && (a.Type3 == "yes");
            Func<Assignment, bool> rejectRule8 = a => a.Related4 == "no" && (a.Type4 == "yes");

            Func<Assignment, bool> rejectRule9 = a => (a.NumPeople1 == "onepeople" || a.NumPeople1 == "manypeople") && a.Type1 == "noperson";
            Func<Assignment, bool> rejectRule10 = a => (a.NumPeople2 == "onepeople" || a.NumPeople2 == "manypeople") && a.Type2 == "noperson";
            Func<Assignment, bool> rejectRule11 = a => (a.NumPeople3 == "onepeople" || a.NumPeople3 == "manypeople") && a.Type3 == "noperson";
            Func<Assignment, bool> rejectRule12 = a => (a.NumPeople4 == "onepeople" || a.NumPeople4 == "manypeople") && a.Type4 == "noperson";

            Func<Assignment, bool> profRejectRule = a => rejectRule1(a) || rejectRule2(a) || rejectRule3(a) || rejectRule4(a);
            Func<Assignment, bool> typeRejectRule = a => rejectRule5(a) || rejectRule6(a) || rejectRule7(a) || rejectRule8(a);
            Func<Assignment, bool> noPeopleRejectRule = a => rejectRule9(a) || rejectRule10(a) || rejectRule11(a) || rejectRule12(a);

            
            // Exept the formal question, if turkers do dont answer more than 4 questions they should be rejected
            var blankRejects = Assignments
                .Where(a => a.Fields
                .Where(f => f.Key.Contains("Answer") && !f.Key.Contains("feeback") && !f.Key.Contains("familiarity") && !f.Key.Contains("domore") && !f.Key.Contains("formal"))
                .Where(f => String.IsNullOrEmpty(f.Value)).Count() >= 4)
                .Select(a => a.AssignmentId + ",blankRule")
                .ToList();

            
            var profRejects = Assignments.Where(a => profRejectRule(a)).Select(a => a.AssignmentId + ",ProfRule").ToList();
            var typeRejects = Assignments.Where(a => typeRejectRule(a)).Select(a => a.AssignmentId + ",TypeRule").ToList();
            var noPeopleRejects = Assignments.Where(a => noPeopleRejectRule(a)).Select(a => a.AssignmentId + ",noPeopleRule").ToList();
            
            var header = new string[] { "AssignmentId,RejectRuleName" };
            
            File.WriteAllLines(resultFile, header.Concat(profRejects).Concat(typeRejects).Concat(noPeopleRejects).Concat(blankRejects));
        }

        public void WriteRejectMessage(string originalFile, string rejectFile, string outputFile)
        {
            string rejectMsg1 = "Your HIT has been checked twice, first automatically and then manually by an expert. We have determined that you have skipped answering one or more questions for no explainable reason. Since we cannot use incomplete assignments, we are rejecting your HIT.";
            string rejectMsg2 = "Your HIT has been checked twice, first automatically and then manually by an expert. We have established that there are one or more inconsistencies in your answers to the questions in the HITs for which there is no explainable reason. We are rejecting your HIT, because of these inconsistencies.";
            
            var assIds = File.ReadAllLines(rejectFile).ToCsvDictionary()
                .Select(i => new { Rule = i["RejectRule"], AssignmentID = i["AssignmentID"] });

            var rulePriorities = new Dictionary<string, string>() 
            { 
                {"Blank", "1"},
                {"NoPeople", "2"},
                {"Prof", "3"},
                {"FashionFocused", "4"}
            };

            var rejectIds = assIds.Join(rulePriorities, r => r.Rule, p => p.Key, (r, p) => new { AssignmentId = r.AssignmentID, Priority = p.Value });
            
            var processedLines = File.ReadAllLines(originalFile).Skip(1).ToList().Select(line =>
            {
                string reject = rejectIds.Where(r => line.Contains(r.AssignmentId))
                    .Select(r => { return r.Priority; })
                    .OrderBy(p => p)
                    .TakeTop(1)
                    .FirstOrDefault();

                if (reject == "1")
                    return String.Format("{0},\"\",\"{1}\"", line, rejectMsg1);
                else if (reject == "2")
                    return String.Format("{0},\"\",\"{1}\"", line, rejectMsg2);
                else if (reject == "3")
                    return String.Format("{0},\"\",\"{1}\"", line, rejectMsg2);
                else if (reject == "4")
                    return String.Format("{0},\"\",\"{1}\"", line, rejectMsg2);
                else if (!String.IsNullOrEmpty(line) && line.Last() == '"')
                    return String.Format("{0},\"x\",\"\"", line);
                else
                    return line;
            });

            File.WriteAllLines(outputFile, processedLines);

        }

        /// <summary>
        /// Create a csv file for the input of AMT HIT
        /// </summary>
        /// <param name="inputFile">Csv file of images and categories</param>
        /// <param name="wikiFile">Csv file of wiki definitions</param>
        /// <param name="outputFile">Output file to save result</param>
        public void CreateHitInputFiles(string inputFile, string wikiFile, string outputFile)
        {
            int recordNo = 1;
            int anchor = 4;
            char sep = ',';

            // Read wiki definitions from wiki definitions files
            var wikiDefs = File.ReadAllLines(wikiFile).ToCsvDictionary('|')
                .Select(i => new
                {
                    Category = i["Category"],
                    SampleImage = i["SampleImage"],
                    Wiki = String.Format("\"{0}\"", i["WikiDef"].Replace("\"","\"\""))
                });

            // 1) Read original images, 
            // 2) Group based on categories,
            // 3) Put each 4 image in 1 record
            // 4) Join with wiki definition file for each category
            // 5) Output to csv
            var output = File.ReadAllLines(inputFile).ToCsvDictionary()
                .Select(i => new { Category = i["Category"], Url = i["Url"] })
                .GroupBy(i => i.Category).Select(g =>
                {
                    int count = 0;
                    return new
                    {
                        Category = g.Key,
                        Urls = g.Aggregate("",
                            // Concatenate urls of each group with format (with ANCHOR 3): "ur1,url2,url3 url4,url5,url6"
                            (res, i) => res += i.Url + ((count++ % anchor == 0) ? " " : sep.ToString()),
                            res => res.Substring(0, res.Length - 1)).Split(' ')
                    };
                })
                .SelectMany(c => c.Urls.Select(u => new { Category = c.Category, Urls = u, UrlCount = u.Split(sep).Length }))
                .Where(i => i.UrlCount == anchor)
                .Join(wikiDefs, i => i.Category, w => w.Category, (i, w) =>
                {
                    int urlCount = i.UrlCount;

                    // Generate Url css classes based on number of urls in each record. 
                    // e.g. with urlCount = 2 : "visible,visible,invisible,invisible"
                    string urlClasses = Enumerable.Range(0, anchor)
                        .Select(j => (j < urlCount) ? "visible" : "invisible")
                        .Aggregate((res, next) => res + sep + next);

                    string template = String.Join(sep.ToString(), "{0},{1},{2},{3},{4},{5}".Split(','));
                    return String.Format(template,
                        recordNo++,
                        i.Category,
                        w.Wiki,
                        w.SampleImage,
                        urlCount,
                        i.Urls
                    );
                });


            string[] fields = "ID,Category,WikiDefinition,SampleImage,UrlCount,Url1,Url2,Url3,Url4".Split(',');
            var header = new String[] { String.Join(sep.ToString(), fields) };

            File.WriteAllLines(outputFile, header.Concat(output));
        }


        public void SplitInputFile(string inputFile, int num, string output1, string output2)
        {
            var lines = File.ReadAllLines(inputFile).ToList();
            var header = lines.Take(1);

            var shuffle = lines.Skip(1).Shuffle();
            var out1 = shuffle.Take(num);
            var out2 = shuffle.Skip(num);

            File.WriteAllLines(output1, header.Concat(out1));
            File.WriteAllLines(output2, header.Concat(out2));
        }


        public void GetNewCategories(string wikiFile, string catsFile, string outputFile)
        {
            // Read wiki definitions from wiki definitions files
            var wikiCats = File.ReadAllLines(wikiFile).ToCsvDictionary('|')
                .Select(i => i["Category"]);

            var newCats = File.ReadAllLines(catsFile).ToCsvDictionary()
                .Select(i => i["Category"])
                .Where(i => !wikiCats.Contains(i));

            File.WriteAllLines(outputFile, newCats);
        }


        public void ProcessCategories(string inputFile, string outputFile, string otherCatFile)
        { 
            var lines = File.ReadAllLines(inputFile).ToList();
            
            File.WriteAllLines(outputFile, lines.Where(l => !String.IsNullOrWhiteSpace(l)).Where(l => !l.StartsWith("*")));
            File.WriteAllLines(otherCatFile, lines.Where(l => l.StartsWith("*")));
        }

        public void ConfirmCategories(string catFiles, string datasetPath, string confirmedCatFile)
        {
            var cats = File.ReadAllLines(catFiles).ToCsvDictionary('|').Select(i => i["Category"]);
            var allCats = Directory.GetDirectories(datasetPath).Select(d => d.Substring(d.LastIndexOf('\\') + 1));

            File.WriteAllLines(confirmedCatFile, cats.Where(c => allCats.Contains(c)));
        }


        /// <summary>
        /// Apply rejection to the result of AMT Hits
        /// </summary>
        /// <param name="rejectFile">A csv file specifiying which Assignments should be rejectedd</param>
        /// <param name="originalFile">Original result file downloaded from AMT</param>
        /// <param name="outputFile">Result of applying rejection which can be uploaded to AMT</param>
        public void ApplyRejection(string rejectFile, string originalFile, string outputFile)
        {
            string reject1 = "The assignments were automatically filtered and then manually and independently reviewed by two experts. There is at least one obligatory question that is not answered in your assignment.";
            string reject2 = "The Assignments were automatically filtered and then manually and independently reviewed by two experts. Both of reviewers agreed that there is mismatch between the answers of the first and second question for one or more images.";

            var rejectList = File.ReadAllLines(rejectFile).Skip(1).Select(line =>
            {
                string[] parts = line.Split(',');
                return new { AssignmentId = parts[0], RejectReason = parts[1] };
            });

            var processedLines = File.ReadAllLines(originalFile).Skip(1).ToList().Select(line =>
            {
                string reject = rejectList.Where(item => line.Contains(item.AssignmentId)).Select(item => { return item.RejectReason; }).FirstOrDefault();

                if (reject == "1")
                    return String.Format("{0},\"\",\"{1}\"", line, reject1);
                else if (reject == "2")
                    return String.Format("{0},\"\",\"{1}\"", line, reject2);
                else if (line.Last() == '"')
                    return String.Format("{0},\"x\",\"\"", line);
                else
                    return line;
            });

            File.WriteAllLines(outputFile, processedLines);
        }

        
        /// <summary>
        /// Get the answers of turkers from each HIT and save them as per image csv file
        /// </summary>
        /// <param name="inputFile">Original AMT result file</param>
        /// <param name="outputFile"></param>
        public void GetAnswersPerImage(string inputFile, string outputFile)
        {
            //var fields = File.ReadAllLines(inputFile).Take(1).FirstOrDefault().Split('|')
            //    .Select((item, index) => new { Index = index, Name = item }).ToDictionary(item => item.Name, item => item.Index);

            var content = new CsvReader(new StreamReader(inputFile), true).ToCsvDictionary()
                .Select(a => new Assignment(a))
                .SelectMany(a => Enumerable.Range(1, a.UrlCount).Select(i => new ImageInfo(a, i)))
                .Where(ii => ii.Status != "Rejected")
                .GroupBy(ii => ii.Url)
                .Where(ii => ii.Count() >= 3)
                .Select(g => new ImageInfo
                {
                    Url = g.Key,
                    Related = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.Related : i.Related),
                    Specialty = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.Specialty : i.Specialty),
                    Familiarity = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.Familiarity : i.Familiarity),
                    NumPeople = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.NumPeople : i.NumPeople),
                    FashionFocused = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.FashionFocused : i.FashionFocused),
                    Prof = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.Prof : i.Prof),
                    Formal = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.Formal : i.Formal),
                    Category = g.Take(3).Select(i => i.Category).FirstOrDefault(),
                    WorkerId = g.Take(3).Aggregate(" ", (current, i) => current != " " ? current + "," + i.WorkerId : i.WorkerId),
                }.ToStringWithMajorityVotes());

            List<string> output = new List<string>() { "PictureURL,Category,T1_Q1,T2_Q1,T3_Q1,Majority Q1,T1_Q2,T2_Q2,T3_Q2,Majority Q2,T1_Q3,T2_Q3,T3_Q3,Majority Q3,"
                + "T1_Q4,T2_Q4,T3_Q4,Majority Q4,T1_Q5,T2_Q5,T3_Q5,Majority Q5,T1_Q6,T2_Q6,T3_Q6,Majority Q6,T1_Familiarity,T2_Familiarity,T3_Familiarity,T1,T2,T3" };
            File.WriteAllLines(outputFile, output.Concat(content));
        }

        /// <summary>
        /// Get statistic of turkers answers for each image
        /// </summary>
        /// <param name="inputFile">Oringinal AMT result file</param>
        /// <param name="outputFile"></param>
        public void GetAnswersStatisticsPerImage(string inputFile, string outputFile)
        {
            var fields = File.ReadAllLines(inputFile).Take(1).FirstOrDefault().Split('|')
                .Select((item, index) => new { Index = index, Name = item }).ToDictionary(item => item.Name, item => item.Index);

            var content = File.ReadAllLines(inputFile).Skip(1)
                .Select(line => new Assignment(line.Split('|'), fields))
                .SelectMany(a => Enumerable.Range(1, a.UrlCount).Select(i => new ImageInfo(a, i)))
                //.Where(ii => ii.Approved)
                .GroupBy(ii => ii.Url)
                .Where(ii => ii.Count() == 3)
                .Select(g => String.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
                    g.Key,
                    g.Select(i => i.Category).FirstOrDefault(),
                    g.Where(i => i.Related == "Yes").Count(),
                    g.Where(i => i.Related == "No").Count(),
                    g.Where(i => i.Related == "NotSure").Count(),
                    g.Where(i => i.Specialty == "Yes").Count(),
                    g.Where(i => i.Specialty == "No").Count(),
                    g.Where(i => i.Specialty == "NotSure").Count()
                ));

            List<string> output = new List<string>() { "PictureURL,Category,Q1CountYes,Q1CountNo,Q1CountNotsure,Q2CountYes,Q2CountNo,Q2CountNotsure" };
            File.WriteAllLines(outputFile, output.Concat(content));
        }

        /// <summary>
        /// Filter non-experts annotations and get only those which are annotated also by experts
        /// </summary>
        /// <param name="nonExpertResults">PerImage statistic csv file for non-experts</param>
        /// <param name="expertResults">PerImage statistic csv file for Experts</param>
        /// <param name="outputFile"></param>
        public void GetSameNonExpertAnnotationsForExpertAnnotations(string nonExpertResults, string expertResults, string outputFile)
        {
            var expertAnnotatedUrls = File.ReadAllLines(expertResults).Skip(1).Select(line => line.Split(',')[0]);

            var joinedUrls = File.ReadAllLines(nonExpertResults).Skip(1)
                .Join(expertAnnotatedUrls, line => line.Split(',')[0], item => item, (line, url) => line);

            File.WriteAllLines(outputFile, File.ReadAllLines(nonExpertResults).Take(1).Concat(joinedUrls));
        }

        /// <summary>
        /// Calculate majority voting for each image
        /// </summary>
        /// <param name="inputFile">PerImage statistic file</param>
        /// <param name="outputFile">Majority Voting output file path</param>
        public void CalculateMajorityVoting(string inputFile, string outputFile)
        {
            var lines = File.ReadAllLines(inputFile);

            var output = lines.Skip(1).Select(line => new { Votes = new Votes(line.Split(',')), Data = line })
                .Select(item => String.Format("{0},{1},{2}", item.Data, item.Votes.MajorityQ1(), item.Votes.MajorityQ2()));

            var header = lines.Take(1).Select(line => line + "Q1_Majority,Q2_Majority");

            File.WriteAllLines(outputFile, header.Concat(output));
        }

        /// <summary>
        /// Merge the majority voting results of experts and non experts
        /// </summary>
        /// <param name="expertFile">Majority voting file for experts</param>
        /// <param name="nonExpertFile">Majority voting file for non-experts</param>
        /// <param name="outputFile"></param>
        public void MergeMajorityVotings(string expertFile, string nonExpertFile, string outputFile)
        {
            var expertVotes = File.ReadAllLines(expertFile).Skip(1).Select(line => line.Split(','))
                .Select(fields => new { Url = fields[0], Q1Majority = fields[10], Q2Majority = fields[11] });

            var nonExpertVotes = File.ReadAllLines(nonExpertFile).Skip(1).Select(line => line.Split(','))
                .Select(fields => new { Url = fields[0], Q1Majority = fields[10], Q2Majority = fields[11] });

            var allVotes = expertVotes.Join(nonExpertVotes, e => e.Url, ne => ne.Url, (e, ne) =>
                String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12}",
                    e.Url,
                    e.Q1Majority,
                    ne.Q1Majority,
                    Convert.ToInt32(e.Q1Majority == "Yes") + Convert.ToInt32(ne.Q1Majority == "Yes"),
                    Convert.ToInt32(e.Q1Majority == "No") + Convert.ToInt32(ne.Q1Majority == "No"),
                    Convert.ToInt32(e.Q1Majority == "Notsure") + Convert.ToInt32(ne.Q1Majority == "Notsure"),
                    Convert.ToInt32(e.Q1Majority == "NoAgreement") + Convert.ToInt32(ne.Q1Majority == "NoAgreement"),

                    e.Q2Majority,
                    ne.Q2Majority,
                    Convert.ToInt32(e.Q2Majority == "Yes") + Convert.ToInt32(ne.Q2Majority == "Yes"),
                    Convert.ToInt32(e.Q2Majority == "No") + Convert.ToInt32(ne.Q2Majority == "No"),
                    Convert.ToInt32(e.Q2Majority == "Notsure") + Convert.ToInt32(ne.Q2Majority == "Notsure"),
                    Convert.ToInt32(e.Q2Majority == "NoAgreement") + Convert.ToInt32(ne.Q2Majority == "NoAgreement")
                )
            );

            var header = new List<string> { 
                "Url,Q1ExpertVote,Q1NonExpertVote,Q1CountYes,Q1CountNo,Q1CountNotsure,Q1CountNoAgreement," +
                "Q2ExpertVote,Q2NonExpertVote,Q2CountYes,Q2CountNo,Q2CountNotsure,Q2CountNoAgreement"
            };

            File.WriteAllLines(outputFile, header.Concat(allVotes));
        }

        /// <summary>
        /// Get some basic statics from original AMT result file
        /// </summary>
        /// <param name="inputFile">Original AMT result file in bsv (bar separated value) format</param>
        /// <param name="outputFile">Output statistics file in tsv (tab separated value) format</param>
        public void GetDataStatistic(string inputFile, string outputFile)
        {
            var output = File.ReadAllLines(inputFile).ToCsvDictionary('|')
                .Select(i => new { WorkerId = i["WorkerId"], Duration = int.Parse(i["WorkTimeInSeconds"]) })
                .GroupBy(i => i.WorkerId)
                .Select(g => new { WorkerId = g.Key, Count = g.Count(), Avg = g.Average(i => i.Duration) })
                .OrderByDescending(i => i.Count)
                .Select(i => String.Format("{0}\t{1}\t{2}", i.WorkerId, i.Count, i.Avg));

            File.WriteAllLines(outputFile, output);
        }

        /// <summary>
        /// Add photoID from database file to AMT result file
        /// </summary>
        /// <param name="photoInfoFile">Path to photo info file</param>
        /// <param name="amtResult">AMT PerImage result file</param>
        /// <param name="outputFile">OutputFile</param>
        public void MergePhotoInfo(string photoInfoFile, string amtResult, string outputFile)
        {
            var photoInfos = File.ReadAllLines(photoInfoFile).ToCsvDictionary('|')
                .Select(i => new { Id = i["PHOTO_ID"], Url = i["MEDIUM_URL"] });

            var output = File.ReadAllLines(amtResult).Skip(1)
                .Select(line => String.Format("{0},{1}",
                    photoInfos.Where(pi => line.Contains(pi.Url)).Select(pi => pi.Id).FirstOrDefault(),
                    line));

            var header = File.ReadAllLines(amtResult).Take(1).Select(line => "PHOTO_ID," + line);

            File.WriteAllLines(outputFile, header.Concat(output));
        }

        public void ReplaceWorkerIdWithIndex(string amtResultFile, string outputFile)
        {
            var header = File.ReadAllLines(amtResultFile).Take(1).ToList();
            var workerIndex = new Dictionary<String, int>();
            int index = 1;

            var output = new CsvReader(new StreamReader(amtResultFile), true).ToCsvDictionary()
                .Select(i =>
                    {
                        if (!workerIndex.ContainsKey(i["WorkerId"]))
                            workerIndex[i["WorkerId"]] = index++;

                        return header.Single().Replace("\"", "").Split(',')
                            .Select(f => f == "WorkerId" ? workerIndex[i["WorkerId"]].ToString() : i[f].Replace("\"", "\"\""))
                            .Aggregate((cur, next) => cur + ",\"" + next + "\"");
                    }).ToList();
            
            header = header.Select(h => h.Replace("WorkerId", "WorkerIndex")).ToList();
            File.WriteAllLines(outputFile, header.Concat(output));
        }

    }

    public class Votes
    {
        public int Q1Yes { get; set; }
        public int Q1No { get; set; }
        public int Q1Notsure { get; set; }

        public int Q2Yes { get; set; }
        public int Q2No { get; set; }
        public int Q2Notsure { get; set; }

        public Votes(string[] input)
        {
            Q1Yes = int.Parse(input[2]);
            Q1No = int.Parse(input[3]);
            Q1Notsure = int.Parse(input[4]);

            Q2Yes = int.Parse(input[5]);
            Q2No = int.Parse(input[6]);
            Q2Notsure = int.Parse(input[7]);
        }

        public string MajorityQ1()
        {
            if (Q1Yes >= 2)
                return "Yes";
            else if (Q1No >= 2)
                return "No";
            else if (Q1Notsure >= 2)
                return "Notsure";
            else
                return "NoAgreement";
        }

        public string MajorityQ2()
        {
            if (Q2Yes >= 2)
                return "Yes";
            else if (Q2No >= 2)
                return "No";
            else if (Q2Notsure >= 2)
                return "Notsure";
            else
                return "NoAgreement";
        }
    }

    public class ImageInfo
    {
        public string HitId { get; set; }
        public string Url { get; set; }
        public string Related { get; set; }
        public string Specialty { get; set; }
        public string Category { get; set; }
        public string Familiarity { get; set; }
        public string Formal { get; set; }
        public string Prof { get; set; }
        public string FashionFocused { get; set; }
        public string NumPeople { get; set; }
        public bool Approved { get; set; }
        public string Status { get; set; }
        public string WorkerId { get; set; }

        public ImageInfo()
        {

        }

        public ImageInfo(Assignment a, int imageNo)
        {
            Category = a.Category;
            Familiarity = a.Familiarity;
            HitId = a.HitId;
            Approved = a.Approved;
            Status = a.Status;
            WorkerId = a.WorkerId;

            switch (imageNo)
            {
                case 1:
                    Url = a.Url1;
                    Related = a.Related1;
                    Specialty = a.Specialty1;
                    Formal = a.Formal1;
                    Prof = a.Prof1;
                    FashionFocused = a.Type1;
                    NumPeople = a.NumPeople1;
                    break;
                case 2:
                    Url = a.Url2;
                    Related = a.Related2;
                    Specialty = a.Specialty2;
                    Formal = a.Formal2;
                    Prof = a.Prof2;
                    FashionFocused = a.Type2;
                    NumPeople = a.NumPeople2;
                    break;
                case 3:
                    Url = a.Url3;
                    Related = a.Related3;
                    Specialty = a.Specialty3;
                    Formal = a.Formal3;
                    Prof = a.Prof3;
                    FashionFocused = a.Type3;
                    NumPeople = a.NumPeople3;
                    break;
                case 4:
                    Url = a.Url4;
                    Related = a.Related4;
                    Specialty = a.Specialty4;
                    Formal = a.Formal4;
                    Prof = a.Prof4;
                    FashionFocused = a.Type4;
                    NumPeople = a.NumPeople4;
                    break;
            }
        }

        public string ToString()
        {
            return String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}", Url, Category, Related, Specialty, NumPeople, FashionFocused, Prof, Formal, Familiarity, WorkerId);
        }

        public string ToStringWithMajorityVotes()
        {
            return String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15}",
                Url, Category, 
                Related, Related.Majority(), 
                Specialty, Specialty.Majority(), 
                NumPeople, NumPeople.Majority(),
                FashionFocused, FashionFocused.Majority(),
                Prof, Prof.Majority(),
                Formal, Formal.Majority(),
                Familiarity,
                WorkerId);
        }
    }

    public static class StringExtenstions
    {
        public static string Majority(this string str)
        {
            return str.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
                .GroupBy(v => v)
                .Where(g => g.Count() >= 2)
                .Select(g => g.Key)
                .FirstOrDefault() ?? "N/A";
        }
    }

    public class Assignment
    {
        public int UrlCount { get; set; }
        public string HitId { get; set; }
        public string WorkerId { get; set; }
        public string AssignmentId { get; set; }
        public string Status { get; set; }

        public string Url1 { get; set; }
        public string Url2 { get; set; }
        public string Url3 { get; set; }
        public string Url4 { get; set; }

        public string Related1 { get; set; }
        public string Related2 { get; set; }
        public string Related3 { get; set; }
        public string Related4 { get; set; }

        public string Specialty1 { get; set; }
        public string Specialty2 { get; set; }
        public string Specialty3 { get; set; }
        public string Specialty4 { get; set; }

        public string Prof1 { get; set; }
        public string Prof2 { get; set; }
        public string Prof3 { get; set; }
        public string Prof4 { get; set; }

        public string Type1 { get; set; }
        public string Type2 { get; set; }
        public string Type3 { get; set; }
        public string Type4 { get; set; }

        public string Formal1 { get; set; }
        public string Formal2 { get; set; }
        public string Formal3 { get; set; }
        public string Formal4 { get; set; }

        public string NumPeople1 { get; set; }
        public string NumPeople2 { get; set; }
        public string NumPeople3 { get; set; }
        public string NumPeople4 { get; set; }

        public string Category { get; set; }
        public string Familiarity { get; set; }
        public bool Approved { get; set; }
        public DateTime SubmitTime { get; set; }

        public Dictionary<string, string> Fields { get; private set; }

        private Dictionary<string, int> _fieldIndices;

        public Assignment(string[] input, Dictionary<string, int> fieldIndices)
        {
            _fieldIndices = fieldIndices;
            InitializeData(input);
        }

        private void InitializeData(string[] input)
        {
            UrlCount = Convert.ToInt32(input[GetFiledIndex("Input.UrlCount")]);
            HitId = input[GetFiledIndex("HITId")];
            WorkerId = input[GetFiledIndex("WorkerId")];
            AssignmentId = input[GetFiledIndex("AssignmentId")];
            Approved = true;
            SubmitTime = ConvertToDateTime(input[GetFiledIndex("SubmitTime")]);
            //Approved = input[GetFiledIndex("Approve")] == "x" ? true : false;

            Url1 = input[GetFiledIndex("Input.Url1")];
            Url2 = input[GetFiledIndex("Input.Url2")];
            Url3 = input[GetFiledIndex("Input.Url3")];
            Url4 = input[GetFiledIndex("Input.Url4")];

            Related1 = input[GetFiledIndex("Answer.related1")].ToLower();
            Related2 = input[GetFiledIndex("Answer.related2")].ToLower();
            Related3 = input[GetFiledIndex("Answer.related3")].ToLower();
            Related4 = input[GetFiledIndex("Answer.related4")].ToLower();

            Specialty1 = input[GetFiledIndex("Answer.specialty1")].ToLower();
            Specialty2 = input[GetFiledIndex("Answer.specialty2")].ToLower();
            Specialty3 = input[GetFiledIndex("Answer.specialty3")].ToLower();
            Specialty4 = input[GetFiledIndex("Answer.specialty4")].ToLower();

            Prof1 = input[GetFiledIndex("Answer.prof1")].ToLower();
            Prof2 = input[GetFiledIndex("Answer.prof2")].ToLower();
            Prof3 = input[GetFiledIndex("Answer.prof3")].ToLower();
            Prof4 = input[GetFiledIndex("Answer.prof4")].ToLower();

            Type1 = input[GetFiledIndex("Answer.type1")].ToLower();
            Type2 = input[GetFiledIndex("Answer.type2")].ToLower();
            Type3 = input[GetFiledIndex("Answer.type3")].ToLower();
            Type4 = input[GetFiledIndex("Answer.type4")].ToLower();

            NumPeople1 = input[GetFiledIndex("Answer.numpeople1")].ToLower();
            NumPeople2 = input[GetFiledIndex("Answer.numpeople2")].ToLower();
            NumPeople3 = input[GetFiledIndex("Answer.numpeople3")].ToLower();
            NumPeople4 = input[GetFiledIndex("Answer.numpeople4")].ToLower();

            Formal1 = input[GetFiledIndex("Answer.formal1")].ToLower();
            Formal2 = input[GetFiledIndex("Answer.formal2")].ToLower();
            Formal3 = input[GetFiledIndex("Answer.formal3")].ToLower();
            Formal4 = input[GetFiledIndex("Answer.formal4")].ToLower();

            Familiarity = input[GetFiledIndex("Answer.familiarity")];

            Category = input[GetFiledIndex("Input.Category")];
            Status = input[GetFiledIndex("AssignmentStatus")];
        }

        public Assignment(Dictionary<string, string> input)
        {
            var data = input.Select((i, ix) => new { Index = ix, Field = i.Key, Value = i.Value });
            _fieldIndices = data.ToDictionary(i => i.Field, i => i.Index);
            InitializeData(data.Select(i => i.Value).ToArray());
            Fields = input;
        }

        private int GetFiledIndex(string name)
        {
            return _fieldIndices.Where(f => f.Key == name).Select(f => f.Value).FirstOrDefault();
        }

        private DateTime ConvertToDateTime(string input)
        {
            var parts = input.Split(' ');

            string date = string.Format("{0} {1} {2}", parts[1], parts[2], parts[5]);
            string time = parts[3];

            return Convert.ToDateTime(date + " " + time);
        }
    }


}
